﻿using System;
using System.IO;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.Collections.Specialized;
using System.Threading;
using System.Diagnostics;
using System.ComponentModel;
using System.Web;
using System.Text.RegularExpressions;
using System.Xml;
using System.IO.Compression;
using System.Security.Principal;

using System.Collections;

namespace com.ubosi.spider.core.Net
{
    /// <summary>
    /// 页面抓取的基类
    /// </summary>
    public class NetSpider
    {
        /// <summary>
        /// 用于分离HTML文档中的charset的正则表达式
        /// </summary>
        private Regex _charsetRegx;
        const string META_ENCODING_REGX = @"<meta\s+http-equiv\s*=\s*[\""'\s]?Content-Type\b.*?charset\s*=\s*(?<encodingName>[^\""'\s>]*)";        

        private WebRequestSetting _requestSettings;
        /// <summary>
        /// Web请求设置
        /// </summary>
        public WebRequestSetting RequestSettings
        {
            get { return _requestSettings; }
            set { _requestSettings = value; }
        }
        private IWebProxyContainer _proxyContainer;

        public NetSpider(WebRequestSetting requestSettings, IWebProxyContainer proxyContainer)
        {
            _requestSettings = requestSettings;
            _proxyContainer = proxyContainer;

            _charsetRegx = new Regex(META_ENCODING_REGX, RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.ExplicitCapture);

        }

        /// <summary>
        /// 根据编码名称尝试获得一个编码，如果无法获得返回null
        /// </summary>
        /// <param name="encodingName"></param>
        /// <returns></returns>
        private Encoding TryGetEncoding(string encodingName)
        {
            try
            {
                return Encoding.GetEncoding(encodingName);
            }
            catch
            {
                return null;
            }
        }

        /// <summary>
        /// 根据获得的html数据及response的编码信息尝试获得文档的编码类型
        /// </summary>
        /// <param name="httpData">已得到的html数据</param>
        /// <param name="responseEncodingName">response的编码名称</param>
        /// <param name="responseCharset">response的字符集名称</param>
        /// <returns></returns>
        private Encoding ParseEncoding(byte[] httpData, string responseEncodingName, string responseCharset)
        {
            Encoding encoding = null;

            string asciiHtmlString = Encoding.ASCII.GetString(httpData);
            Match m = _charsetRegx.Match(asciiHtmlString);
            // 文档中存在标有encoding的Meta信息
            if (m.Success)
            {
                string encodingName = m.Groups["encodingName"].Value;
                encoding = TryGetEncoding(encodingName);
            }

            //文档中不存在标有encoding的Meta信息的时候，应用HTTP头的ContentEncoding信息
            if (encoding == null)
            {
                encoding = TryGetEncoding(responseEncodingName);
            }

            //文档中不存在标有encoding的Meta信息的时候，也不存在HTTP头的ContentEncoding信息
            if (encoding == null)
            {
                encoding = TryGetEncoding(responseCharset);
            }

            // 如果找不到相应的Encoding，则使用Default
            if (encoding == null)
                encoding = Encoding.Default;
            return encoding;
        }

        /// <summary>
        /// 从指定的uri路径下载一篇html文档
        /// </summary>
        /// <param name="uri">要从其下载html数据的路径 </param>
        /// <param name="encoding">指定编码</param>
        /// <returns></returns>
        public string GetHtmlDocument(string uri,Encoding encoding)
        {
            uri = HttpUtility.UrlDecode(uri, encoding);

            HttpWebRequest webReq = CreateWebRequest(uri);
            byte[] htmlData = null;

            try
            {
                HttpWebResponse response = webReq.GetResponse() as HttpWebResponse;


                using (BinaryReader br = new BinaryReader(response.GetResponseStream()))
                {
                    int bufferLen = 102400;
                    byte[] buffer = new byte[bufferLen];
                    using (MemoryStream ms = new MemoryStream())
                    {
                        int count = 0;
                        do
                        {
                            count = br.Read(buffer, 0, bufferLen);
                            if (count > 0)
                                ms.Write(buffer, 0, count);
                        } while (count > 0);

                        htmlData = ms.ToArray();
                    }
                }

                //Encoding encoding = ParseEncoding(htmlData, response.ContentEncoding, response.CharacterSet);

                return encoding.GetString(htmlData);
            }
            catch (WebException ex)
            {
                if (_proxyContainer != null && webReq.Proxy != null)
                    _proxyContainer.NotifyError(webReq.Proxy, ex);
                throw ex;
            }
            catch (Exception ex)
            {
                throw ex;
            }
            finally
            {
                if (_proxyContainer != null && webReq.Proxy != null)
                    _proxyContainer.Release(webReq.Proxy);
            }

        }

        /// <summary>
        /// 从指定的uri路径下载一篇html文档，缺省编码
        /// </summary>
        /// <param name="uri">要从其下载html数据的路径</param>
        /// <returns></returns>
        public string GetHtmlDocument(string uri)
        {
            return GetHtmlDocument(uri, Encoding.Default);
        }
        
        /// <summary>
        /// 从指定的uri路径下载一篇xml文档
        /// </summary>
        /// <param name="uri"></param>
        /// <returns></returns>
        public string GetXmlDocument(string uri)
        {
            return GetHtmlDocument(uri, Encoding.UTF8).Trim();            
        }
        /// <summary>
        /// 从文本文件读取Ｈｔｍｌ
        /// </summary>
        /// <param name="fileName"></param>
        /// <param name="encoding"></param>
        /// <returns></returns>
        public string GetHtmlFromFile(string fileName,Encoding encoding)
        {
            try
            {
                return File.ReadAllText(fileName, encoding);
            }
            catch(Exception ex)
            {
                return ex.Message;
            }

        }
        /// <summary>
        /// 下载40K以内的图片
        /// </summary>
        /// <param name="fileName"></param>
        /// <param name="localPath"></param>
        /// <param name="ImageUrl"></param>
        /// <returns></returns>
        public string DownloadImage(string fileName, string localPath, string ImageUrl)
        {
            try
            {
                HttpWebRequest request = CreateWebRequest(ImageUrl);
                System.Net.HttpWebResponse response = null;
                response = (System.Net.HttpWebResponse)request.GetResponse();
                Stream s = response.GetResponseStream();
                BinaryReader br = new BinaryReader(s);
                int length2 = Int32.Parse(response.ContentLength.ToString());
                byte[] byteArr = new byte[length2];
                s.Read(byteArr, 0, length2);
                if (File.Exists(localPath + fileName))
                {
                    File.Delete(localPath + fileName);
                }
                if (!Directory.Exists(localPath))
                {
                    Directory.CreateDirectory(localPath);
                }
                FileStream fs = File.Create(localPath + fileName);
                try
                {
                    fs.Write(byteArr, 0, length2);
                }
                finally
                {
                    fs.Close();
                    br.Close();
                }
                return fileName;
            }
            catch (Exception ex)
            {
                return string.Empty;
            }            
        }
        /// <summary>
        /// 为指定的url路径获取一个webrequest对象
        /// </summary>
        /// <param name="uriString"></param>
        /// <returns></returns>
        private HttpWebRequest CreateWebRequest(string uriString)
        {
            Uri uri = new Uri(uriString);
            if (uri.Scheme != Uri.UriSchemeHttp)
                return null;

            HttpWebRequest req = WebRequest.Create(uri) as HttpWebRequest;

            req.Timeout = _requestSettings.Timeout;
            req.UserAgent = _requestSettings.UserAgent;
            req.Accept = _requestSettings.Accept;
            req.KeepAlive = true;
            if (_requestSettings.Headers != null)
            {
                foreach (KeyValuePair<HttpRequestHeader, string> header in _requestSettings.Headers)
                {
                    req.Headers.Add(header.Key, header.Value);
                }
            }

            req.AutomaticDecompression = _requestSettings.AutomaticDecompression;
            req.AllowAutoRedirect = _requestSettings.AllowAutoRedirect;
            req.Referer = _requestSettings.Referer;
            req.PreAuthenticate = true;
            req.UnsafeAuthenticatedConnectionSharing = true;
            req.CookieContainer = new CookieContainer();
            if (_proxyContainer != null)
                req.Proxy = _proxyContainer.GetProxy();

            return req;
        }

        /// <summary>
        /// 根据编码名称尝试获得一个编码，如果无法获得返回null
        /// </summary>
        /// <param name="encodingName"></param>
        /// <returns></returns>
        public static Encoding GetEncoding(string encodingName)
        {
            try
            {
                return Encoding.GetEncoding(encodingName);
            }
            catch
            {
                return Encoding.Default;
            }
        }
    }
    public class HtmlPage
    {
        public string HtmlDocument { get; set; }
        public Encoding DocumentEncoding { get; set; }
        public string PageTitle { get; set; }
        public string PageKeywords { get; set; }
        public string PageDescription { get; set; }

    }
}
